u32 evt; /* effective virtual time */
int warpback; /* warp? */
int warp; /* warp set and within the warp
- limits*/
+ limits*/
s32 warp_value; /* virtual time warp */
s_time_t warpl; /* warp limit */
struct ac_timer warp_timer; /* deals with warpl */
struct bvt_cpu_info
{
- spinlock_t run_lock; /* protects runqueue */
- struct list_head runqueue; /* runqueue for given processor */
- unsigned long svt; /* XXX check this is unsigned long! */
+ struct list_head runqueue;
+ unsigned long svt;
};
-
#define BVT_INFO(p) ((struct bvt_dom_info *)(p)->sched_priv)
#define CPU_INFO(cpu) ((struct bvt_cpu_info *)(schedule_data[cpu]).sched_priv)
#define RUNLIST(p) ((struct list_head *)&(BVT_INFO(p)->run_list))
#define TIME_SLOP (s32)MICROSECS(50) /* allow time to slip a bit */
static s32 ctx_allow = (s32)MILLISECS(5); /* context switch allowance */
-/* SLAB cache for struct bvt_dom_info objects */
static xmem_cache_t *dom_info_cache;
-/*
- * Wrappers for run-queue management. Must be called with the run_lock
- * held.
- */
static inline void __add_to_runqueue_head(struct domain *d)
{
list_add(RUNLIST(d), RUNQUEUE(d->processor));
static void warp_timer_fn(unsigned long pointer)
{
struct bvt_dom_info *inf = (struct bvt_dom_info *)pointer;
- unsigned long flags;
+ unsigned int cpu = inf->domain->processor;
- spin_lock_irqsave(&CPU_INFO(inf->domain->processor)->run_lock, flags);
+ spin_lock_irq(&schedule_data[cpu].schedule_lock);
+
inf->warp = 0;
+
/* unwarp equal to zero => stop warping */
- if(inf->warpu == 0)
+ if ( inf->warpu == 0 )
{
inf->warpback = 0;
- goto reschedule;
+ cpu_raise_softirq(cpu, SCHEDULE_SOFTIRQ);
}
/* set unwarp timer */
inf->unwarp_timer.expires = NOW() + inf->warpu;
add_ac_timer(&inf->unwarp_timer);
- spin_unlock_irqrestore(&CPU_INFO(inf->domain->processor)->run_lock, flags);
-reschedule:
- cpu_raise_softirq(inf->domain->processor, SCHEDULE_SOFTIRQ);
+ spin_unlock_irq(&schedule_data[cpu].schedule_lock);
}
static void unwarp_timer_fn(unsigned long pointer)
{
- struct bvt_dom_info *inf = (struct bvt_dom_info *)pointer;
- unsigned long flags;
+ struct bvt_dom_info *inf = (struct bvt_dom_info *)pointer;
+ unsigned int cpu = inf->domain->processor;
+
+ spin_lock_irq(&schedule_data[cpu].schedule_lock);
- spin_lock_irqsave(&CPU_INFO(inf->domain->processor)->run_lock, flags);
- if(inf->warpback)
+ if ( inf->warpback )
{
inf->warp = 1;
- cpu_raise_softirq(inf->domain->processor, SCHEDULE_SOFTIRQ);
+ cpu_raise_softirq(cpu, SCHEDULE_SOFTIRQ);
}
-
- spin_unlock_irqrestore(&CPU_INFO(inf->domain->processor)->run_lock, flags);
+
+ spin_unlock_irq(&schedule_data[cpu].schedule_lock);
}
-
-
static inline u32 calc_avt(struct domain *d, s_time_t now)
{
u32 ranfor, mcus;
return inf->avt + mcus * inf->mcu_advance;
}
-
/*
* Calculate the effective virtual time for a domain. Take into account
* warping limits
*/
static inline u32 calc_evt(struct domain *d, u32 avt)
{
- struct bvt_dom_info *inf = BVT_INFO(d);
- /* TODO The warp routines need to be rewritten GM */
+ struct bvt_dom_info *inf = BVT_INFO(d);
+ /* TODO The warp routines need to be rewritten GM */
if ( inf->warp )
return avt - inf->warp_value;
*
* Returns non-zero on failure.
*/
-int bvt_alloc_task(struct domain *p)
+int bvt_alloc_task(struct domain *d)
{
- p->sched_priv = xmem_cache_alloc(dom_info_cache);
- if ( p->sched_priv == NULL )
+ if ( (d->sched_priv = xmem_cache_alloc(dom_info_cache)) == NULL )
return -1;
-
+ memset(d->sched_priv, 0, sizeof(struct bvt_dom_info));
return 0;
}
/*
* Add and remove a domain
*/
-void bvt_add_task(struct domain *p)
+void bvt_add_task(struct domain *d)
{
- struct bvt_dom_info *inf = BVT_INFO(p);
+ struct bvt_dom_info *inf = BVT_INFO(d);
ASSERT(inf != NULL);
- ASSERT(p != NULL);
+ ASSERT(d != NULL);
inf->mcu_advance = MCU_ADVANCE;
- inf->domain = p;
+ inf->domain = d;
inf->warpback = 0;
/* Set some default values here. */
inf->warp = 0;
inf->warpu = MILLISECS(1000);
/* initialise the timers */
init_ac_timer(&inf->warp_timer);
- inf->warp_timer.cpu = p->processor;
+ inf->warp_timer.cpu = d->processor;
inf->warp_timer.data = (unsigned long)inf;
inf->warp_timer.function = &warp_timer_fn;
init_ac_timer(&inf->unwarp_timer);
- inf->unwarp_timer.cpu = p->processor;
+ inf->unwarp_timer.cpu = d->processor;
inf->unwarp_timer.data = (unsigned long)inf;
inf->unwarp_timer.function = &unwarp_timer_fn;
- if ( p->domain == IDLE_DOMAIN_ID )
+ if ( d->domain == IDLE_DOMAIN_ID )
{
inf->avt = inf->evt = ~0U;
}
else
{
/* Set avt and evt to system virtual time. */
- inf->avt = CPU_SVT(p->processor);
- inf->evt = CPU_SVT(p->processor);
- }
-
- return;
+ inf->avt = CPU_SVT(d->processor);
+ inf->evt = CPU_SVT(d->processor);
+ }
}
int bvt_init_idle_task(struct domain *p)
{
- unsigned long flags;
-
- if(bvt_alloc_task(p) < 0) return -1;
+ if ( bvt_alloc_task(p) < 0 )
+ return -1;
bvt_add_task(p);
- spin_lock_irqsave(&CPU_INFO(p->processor)->run_lock, flags);
-
set_bit(DF_RUNNING, &p->flags);
if ( !__task_on_runqueue(p) )
__add_to_runqueue_head(p);
- spin_unlock_irqrestore(&CPU_INFO(p->processor)->run_lock, flags);
-
return 0;
}
void bvt_wake(struct domain *d)
{
- unsigned long flags;
struct bvt_dom_info *inf = BVT_INFO(d);
struct domain *curr;
s_time_t now, r_time;
int cpu = d->processor;
u32 curr_evt;
- /* The runqueue accesses must be protected */
- spin_lock_irqsave(&CPU_INFO(cpu)->run_lock, flags);
-
- /* If on the runqueue already then someone has done the wakeup work. */
if ( unlikely(__task_on_runqueue(d)) )
- {
- spin_unlock_irqrestore(&CPU_INFO(cpu)->run_lock, flags);
return;
- }
__add_to_runqueue_head(d);
now = NOW();
/* Set the BVT parameters. AVT should always be updated
- if CPU migration ocurred.*/
+ if CPU migration ocurred.*/
if ( inf->avt < CPU_SVT(cpu) ||
- unlikely(test_bit(DF_MIGRATED, &d->flags)) )
+ unlikely(test_bit(DF_MIGRATED, &d->flags)) )
inf->avt = CPU_SVT(cpu);
/* Deal with warping here. */
inf->evt = calc_evt(d, inf->avt);
- spin_unlock_irqrestore(&CPU_INFO(cpu)->run_lock, flags);
-
- /* Access to schedule_data protected by schedule_lock */
- spin_lock_irqsave(&schedule_data[cpu].schedule_lock, flags);
curr = schedule_data[cpu].curr;
curr_evt = calc_evt(curr, calc_avt(curr, now));
/* Calculate the time the current domain would run assuming
the second smallest evt is of the newly woken domain */
r_time = curr->lastschd +
- ((inf->evt - curr_evt) / BVT_INFO(curr)->mcu_advance) +
- ctx_allow;
+ ((inf->evt - curr_evt) / BVT_INFO(curr)->mcu_advance) +
+ ctx_allow;
if ( is_idle_task(curr) || (inf->evt <= curr_evt) )
cpu_raise_softirq(cpu, SCHEDULE_SOFTIRQ);
else if ( schedule_data[cpu].s_timer.expires > r_time )
mod_ac_timer(&schedule_data[cpu].s_timer, r_time);
-
- spin_unlock_irqrestore(&schedule_data[cpu].schedule_lock, flags);
}
static void bvt_sleep(struct domain *d)
{
- unsigned long flags;
-
if ( test_bit(DF_RUNNING, &d->flags) )
cpu_raise_softirq(d->processor, SCHEDULE_SOFTIRQ);
- else
- {
- /* The runqueue accesses must be protected */
- spin_lock_irqsave(&CPU_INFO(d->processor)->run_lock, flags);
-
-
- if ( __task_on_runqueue(d) )
- __del_from_runqueue(d);
-
- spin_unlock_irqrestore(&CPU_INFO(d->processor)->run_lock, flags);
- }
+ else if ( __task_on_runqueue(d) )
+ __del_from_runqueue(d);
}
/**
* bvt_free_task - free BVT private structures for a task
- * @p: task
+ * @d: task
*/
-void bvt_free_task(struct domain *p)
-{
- ASSERT( p->sched_priv != NULL );
- xmem_cache_free( dom_info_cache, p->sched_priv );
-}
-
-
-/*
- * Block the currently-executing domain until a pertinent event occurs.
- */
-static void bvt_do_block(struct domain *p)
+void bvt_free_task(struct domain *d)
{
+ ASSERT(d->sched_priv != NULL);
+ xmem_cache_free(dom_info_cache, d->sched_priv);
}
/* Control the scheduler. */
struct bvt_ctl *params = &cmd->u.bvt;
if ( cmd->direction == SCHED_INFO_PUT )
- {
ctx_allow = params->ctx_allow;
- }
else
- {
params->ctx_allow = ctx_allow;
- }
return 0;
}
/* Adjust scheduling parameter for a given domain. */
-int bvt_adjdom(struct domain *p,
- struct sched_adjdom_cmd *cmd)
+int bvt_adjdom(
+ struct domain *d, struct sched_adjdom_cmd *cmd)
{
struct bvt_adjdom *params = &cmd->u.bvt;
- unsigned long flags;
-
+
if ( cmd->direction == SCHED_INFO_PUT )
{
u32 mcu_adv = params->mcu_adv;
s_time_t warpl = params->warpl;
s_time_t warpu = params->warpu;
- struct bvt_dom_info *inf = BVT_INFO(p);
+ struct bvt_dom_info *inf = BVT_INFO(d);
DPRINTK("Get domain %u bvt mcu_adv=%u, warpback=%d, warpvalue=%d, "
"warpl=%lld, warpu=%lld\n",
- p->domain, inf->mcu_advance, inf->warpback, inf->warp_value,
+ d->domain, inf->mcu_advance, inf->warpback, inf->warp_value,
inf->warpl, inf->warpu);
/* Sanity -- this can avoid divide-by-zero. */
- if ( mcu_adv == 0 )
- {
- printk("Mcu advance must not be set to 0 (domain %d)\n",p->domain);
+ if ( (mcu_adv == 0) || (warpl < 0) || (warpu < 0) )
return -EINVAL;
- }
- else if ( warpl < 0 || warpu < 0)
- {
- printk("Warp limits must be >= 0 (domain %d)\n", p->domain);
- return -EINVAL;
- }
-
-
- spin_lock_irqsave(&CPU_INFO(p->processor)->run_lock, flags);
+
inf->mcu_advance = mcu_adv;
inf->warpback = warpback;
/* The warp should be the same as warpback */
/* If the unwarp timer set up it needs to be removed */
rem_ac_timer(&inf->unwarp_timer);
/* If we stop warping the warp timer needs to be removed */
- if(!warpback)
+ if ( !warpback )
rem_ac_timer(&inf->warp_timer);
DPRINTK("Get domain %u bvt mcu_adv=%u, warpback=%d, warpvalue=%d, "
"warpl=%lld, warpu=%lld\n",
- p->domain, inf->mcu_advance, inf->warpback, inf->warp_value,
+ d->domain, inf->mcu_advance, inf->warpback, inf->warp_value,
inf->warpl, inf->warpu);
- spin_unlock_irqrestore(&CPU_INFO(p->processor)->run_lock, flags);
}
else if ( cmd->direction == SCHED_INFO_GET )
{
- struct bvt_dom_info *inf = BVT_INFO(p);
-
- spin_lock_irqsave(&CPU_INFO(p->processor)->run_lock, flags);
+ struct bvt_dom_info *inf = BVT_INFO(d);
params->mcu_adv = inf->mcu_advance;
params->warpvalue = inf->warp_value;
params->warpback = inf->warpback;
params->warpl = inf->warpl;
params->warpu = inf->warpu;
- spin_unlock_irqrestore(&CPU_INFO(p->processor)->run_lock, flags);
}
return 0;
*/
static task_slice_t bvt_do_schedule(s_time_t now)
{
- unsigned long flags;
- struct domain *prev = current, *next = NULL, *next_prime, *p;
+ struct domain *prev = current, *next = NULL, *next_prime, *p;
struct list_head *tmp;
int cpu = prev->processor;
s32 r_time; /* time for new dom to run */
u32 next_evt, next_prime_evt, min_avt;
- struct bvt_dom_info *prev_inf = BVT_INFO(prev),
- *p_inf = NULL,
- *next_inf = NULL,
- *next_prime_inf = NULL;
+ struct bvt_dom_info *prev_inf = BVT_INFO(prev);
+ struct bvt_dom_info *p_inf = NULL;
+ struct bvt_dom_info *next_inf = NULL;
+ struct bvt_dom_info *next_prime_inf = NULL;
task_slice_t ret;
-
ASSERT(prev->sched_priv != NULL);
ASSERT(prev_inf != NULL);
- spin_lock_irqsave(&CPU_INFO(cpu)->run_lock, flags);
-
ASSERT(__task_on_runqueue(prev));
if ( likely(!is_idle_task(prev)) )
add_ac_timer(&next_inf->warp_timer);
}
- spin_unlock_irqrestore(&CPU_INFO(cpu)->run_lock, flags);
-
/* Extract the domain pointers from the dom infos */
next = next_inf->domain;
next_prime = next_prime_inf->domain;
/* check for virtual time overrun on this cpu */
if ( CPU_SVT(cpu) >= 0xf0000000 )
{
- u_long t_flags;
-
- write_lock_irqsave(&tasklist_lock, t_flags);
+ ASSERT(!local_irq_is_enabled());
+
+ write_lock(&tasklist_lock);
for_each_domain ( p )
{
}
}
- write_unlock_irqrestore(&tasklist_lock, t_flags);
+ write_unlock(&tasklist_lock);
CPU_SVT(cpu) -= 0xe0000000;
}
static void bvt_dump_cpu_state(int i)
{
- unsigned long flags;
struct list_head *list, *queue;
int loop = 0;
struct bvt_dom_info *d_inf;
struct domain *d;
- spin_lock_irqsave(&CPU_INFO(i)->run_lock, flags);
printk("svt=0x%08lX ", CPU_SVT(i));
queue = RUNQUEUE(i);
printk("QUEUE rq %lx n: %lx, p: %lx\n", (unsigned long)queue,
- (unsigned long) queue->next, (unsigned long) queue->prev);
+ (unsigned long) queue->next, (unsigned long) queue->prev);
list_for_each ( list, queue )
{
d_inf = list_entry(list, struct bvt_dom_info, run_list);
d = d_inf->domain;
printk("%3d: %u has=%c ", loop++, d->domain,
- test_bit(DF_RUNNING, &d->flags) ? 'T':'F');
+ test_bit(DF_RUNNING, &d->flags) ? 'T':'F');
bvt_dump_runq_el(d);
printk("c=0x%X%08X\n", (u32)(d->cpu_time>>32), (u32)d->cpu_time);
printk(" l: %lx n: %lx p: %lx\n",
- (unsigned long)list, (unsigned long)list->next,
- (unsigned long)list->prev);
+ (unsigned long)list, (unsigned long)list->next,
+ (unsigned long)list->prev);
}
- spin_unlock_irqrestore(&CPU_INFO(i)->run_lock, flags);
-}
-
-/* We use cache to create the bvt_dom_infos
- this functions makes sure that the run_list
- is initialised properly.
- Call to __task_on_runqueue needs to return false */
-static void cache_constructor(void *arg1, xmem_cache_t *arg2, unsigned long arg3)
-{
- struct bvt_dom_info *dom_inf = (struct bvt_dom_info*)arg1;
- dom_inf->run_list.next = NULL;
- dom_inf->run_list.prev = NULL;
}
/* Initialise the data structures. */
}
INIT_LIST_HEAD(RUNQUEUE(i));
- spin_lock_init(&CPU_INFO(i)->run_lock);
CPU_SVT(i) = 0; /* XXX do I really need to do this? */
}
- dom_info_cache = xmem_cache_create("BVT dom info",
- sizeof(struct bvt_dom_info),
- 0, 0, cache_constructor, NULL);
-
+ dom_info_cache = xmem_cache_create(
+ "BVT dom info", sizeof(struct bvt_dom_info), 0, 0, NULL, NULL);
if ( dom_info_cache == NULL )
{
printk("BVT: Failed to allocate domain info SLAB cache");
return 0;
}
-
-
struct scheduler sched_bvt_def = {
.name = "Borrowed Virtual Time",
.opt_name = "bvt",
.alloc_task = bvt_alloc_task,
.add_task = bvt_add_task,
.free_task = bvt_free_task,
- .do_block = bvt_do_block,
.do_schedule = bvt_do_schedule,
.control = bvt_ctl,
.adjdom = bvt_adjdom,
.sleep = bvt_sleep,
.wake = bvt_wake,
};
-
/* This is global for now so that private implementations can reach it */
schedule_data_t schedule_data[NR_CPUS];
-/*
- * TODO: It would be nice if the schedulers array could get populated
- * automagically without having to hack the code in here.
- */
-extern struct scheduler sched_bvt_def, sched_fbvt_def, sched_rrobin_def, sched_atropos_def;
-static struct scheduler *schedulers[] = { &sched_bvt_def,
- &sched_fbvt_def,
- &sched_rrobin_def,
- &sched_atropos_def,
- NULL};
+extern struct scheduler sched_bvt_def;
+extern struct scheduler sched_fbvt_def;
+extern struct scheduler sched_rrobin_def;
+extern struct scheduler sched_atropos_def;
+static struct scheduler *schedulers[] = {
+ &sched_bvt_def,
+#ifdef BROKEN_SCHEDULERS
+ &sched_fbvt_def,
+ &sched_rrobin_def,
+ &sched_atropos_def,
+#endif
+ NULL
+};
/* Operations for the current scheduler. */
static struct scheduler ops;
void init_idle_task(void)
{
- struct domain *d = current;
-
- if ( SCHED_OP(init_idle_task, d) < 0)
- panic("Failed to initialise idle task for processor %d",d->processor);
+ if ( SCHED_OP(init_idle_task, current) < 0 )
+ BUG();
}
void domain_sleep(struct domain *d)
{
unsigned long flags;
- /* sleep and wake protected by domain's sleep_lock */
- spin_lock_irqsave(&d->sleep_lock, flags);
+ spin_lock_irqsave(&schedule_data[d->processor].schedule_lock, flags);
+
if ( likely(!domain_runnable(d)) )
SCHED_OP(sleep, d);
- spin_unlock_irqrestore(&d->sleep_lock, flags);
+
+ spin_unlock_irqrestore(&schedule_data[d->processor].schedule_lock, flags);
/* Synchronous. */
while ( test_bit(DF_RUNNING, &d->flags) && !domain_runnable(d) )
void domain_wake(struct domain *d)
{
- unsigned long flags;
+ unsigned long flags;
+
+ spin_lock_irqsave(&schedule_data[d->processor].schedule_lock, flags);
- spin_lock_irqsave(&d->sleep_lock, flags);
-
if ( likely(domain_runnable(d)) )
{
TRACE_2D(TRC_SCHED_WAKE, d->domain, d);
clear_bit(DF_MIGRATED, &d->flags);
- spin_unlock_irqrestore(&d->sleep_lock, flags);
+ spin_unlock_irqrestore(&schedule_data[d->processor].schedule_lock, flags);
}
/* Block the currently-executing domain until a pertinent event occurs. */
return 0;
}
-
/*
* Demultiplex scheduler-related hypercalls.
*/
/* Adjust scheduling parameter for a given domain. */
long sched_adjdom(struct sched_adjdom_cmd *cmd)
{
- struct domain *p;
-
+ struct domain *d;
+
if ( cmd->sched_id != ops.sched_id )
return -EINVAL;
if ( cmd->direction != SCHED_INFO_PUT && cmd->direction != SCHED_INFO_GET )
return -EINVAL;
- p = find_domain_by_id(cmd->domain);
-
- if( p == NULL )
+ d = find_domain_by_id(cmd->domain);
+ if ( d == NULL )
return -ESRCH;
- TRACE_1D(TRC_SCHED_ADJDOM, p->domain);
+ TRACE_1D(TRC_SCHED_ADJDOM, d->domain);
- SCHED_OP(adjdom, p, cmd);
+ spin_lock_irq(&schedule_data[d->processor].schedule_lock);
+ SCHED_OP(adjdom, d, cmd);
+ spin_unlock_irq(&schedule_data[d->processor].schedule_lock);
- put_domain(p);
+ put_domain(d);
return 0;
}
rem_ac_timer(&schedule_data[cpu].s_timer);
ASSERT(!in_irq());
- // TODO - move to specific scheduler ASSERT(__task_on_runqueue(prev));
if ( test_bit(DF_BLOCKED, &prev->flags) )
{
schedule_data[cpu].s_timer.expires = now + r_time;
add_ac_timer(&schedule_data[cpu].s_timer);
+ /* Must be protected by the schedule_lock! */
+ set_bit(DF_RUNNING, &next->flags);
+
spin_unlock_irq(&schedule_data[cpu].schedule_lock);
/* Ensure that the domain has an up-to-date time base. */
* without warning).
*/
clear_bit(DF_RUNNING, &prev->flags);
- set_bit(DF_RUNNING, &next->flags);
/* Mark a timer event for the newly-scheduled domain. */
if ( !is_idle_task(next) )
{
s_time_t now = NOW();
int i;
+ unsigned long flags;
+
+ local_irq_save(flags);
printk("Scheduler: %s (%s)\n", ops.name, ops.opt_name);
SCHED_OP(dump_settings);
printk("NOW=0x%08X%08X\n", (u32)(now>>32), (u32)now);
+
for ( i = 0; i < smp_num_cpus; i++ )
{
+ spin_lock(&schedule_data[i].schedule_lock);
printk("CPU[%02d] ", i);
SCHED_OP(dump_cpu_state,i);
+ spin_unlock(&schedule_data[i].schedule_lock);
}
+
+ local_irq_restore(flags);
}
#if defined(WAKE_HISTO) || defined(BLOCKTIME_HISTO)
projects / xen.git / commitdiff